#!/usr/bin/perl
use Data::Dumper;
my $vocab = {};
foreach my $topic ('food','politics','sports','tech') {
    my @files = split(' ',`ls $topic`);
    foreach my $file (@files) {
	open FILE, "$topic/$file" or die "couldn't open file $file!\n";
	my $line = <FILE>;
	$line = lc($line);
	my @words = split(' ',$line);
	foreach my $word (@words) {
	    next if $word =~ /^'/;
	    next if $word =~ /\d+/;
	    next if $word =~ /'$/;
	    if(!defined($vocab->{$word}->{$file})) {
		$vocab->{$word}->{$file} = 1;
	    }
	    else {
		$vocab->{$word}->{$file}++;
	    }
	}
    }
}
open WL, ">wordlist.txt";
foreach my $word (sort keys %{$vocab}) {
    print WL "$word\n";
}
open TL, ">topiclist.txt";
open WC, ">wordcounts.txt";
my $docct = 1;
foreach my $topic('food','politics','sports','tech') {
    my @files = split(' ',`ls $topic`);
    print TL "$docct $topic\n";
    foreach my $file (@files) {
	print "$topic $file\n";
	$wordct = 1;
	foreach my $word (sort keys %{$vocab}) {
	    if(defined($vocab->{$word}->{$file})) {
		print WC "$docct $wordct $vocab->{$word}->{$file}\n";
	    }
	    $wordct++;
	}
	$docct++;
    }
}

#print Dumper $vocab;
